#database
import os
import chardet
from tinydb import TinyDB, Query
import docx2txt
from xlsx2csv import Xlsx2csv
import pdftotext

#word,excel,ppt,txt,pdf不同类型的文档选择不同的处理方式
def file2txt(file):
    if os.path.basename(file).endswith('.txt') or os.path.basename(file).endswith('.TXT'):
        return txt2txt(file)
    if os.path.basename(file).endswith('.docx') or os.path.basename(file).endswith('.DOCX'):
        return docxtotxt(file)
    if os.path.basename(file).endswith('.xlsx') or os.path.basename(file).endswith('.XLSX'):
        return xlsx2txt(file)
    if os.path.basename(file).endswith('.pptx') or os.path.basename(file).endswith('.PPTX'):
        return pptx2txt(file)
    if os.path.basename(file).endswith('.pdf') or os.path.basename(file).endswith('.PDF'):
        return pdf2txt(file)

    
#TXT文档的处理方式，判断编码类型，统一改编码为UTF-8码
def txt2txt(file):
    with open(file,mode='rb') as f3:
        data = f3.read()
    result = chardet.detect(data)

        
    with open(file,'r',encoding=result['encoding']) as f:
        content=' '.join(f.readlines())
    if "UTF" in str(result['encoding']):
        pass
    else:
        with open(file,'w',encoding='UTF-8') as f:
            f.write(content)

    return {'abspath':os.path.abspath(file),'dirname':os.path.dirname(file),'basename':os.path.basename(file),'path0':os.path.dirname(file)+'/'+os.path.basename(file),'mtime':os.path.getmtime(file),'content':content}

#Word文档的处理方式
def docxtotxt(file):
    content = docx2txt.process(file)

    return {'abspath':os.path.abspath(file),'dirname':os.path.dirname(file),'basename':os.path.basename(file),'path0':os.path.dirname(file)+'/'+os.path.basename(file),'mtime':os.path.getmtime(file),'content':content}

#Xlsx文档的处理方式
def xlsx2txt(file):
    Xlsx2csv(file, outputencoding="utf-8").convert("tmp.txt")
    with open('tmp.txt','r',encoding='utf-8') as f:
        content=' '.join(f.readlines())

    return {'abspath':os.path.abspath(file),'dirname':os.path.dirname(file),'basename':os.path.basename(file),'path0':os.path.dirname(file)+'/'+os.path.basename(file),'mtime':os.path.getmtime(file),'content':content}

#PPTX文档的处理方式
def pptx2txt(file):
    print('python3 -m pptx2md {}'.format(file))
    os.system('python3 -m pptx2md {}'.format(file))
    with open('out.md','r',encoding='utf-8') as f:
        content=' '.join(f.readlines())

    return {'abspath':os.path.abspath(file),'dirname':os.path.dirname(file),'basename':os.path.basename(file),'path0':os.path.dirname(file)+'/'+os.path.basename(file),'mtime':os.path.getmtime(file),'content':content}

#pdf文档的处理方式
def pdf2txt(file):
    with open(file, "rb") as f:
        pdf = pdftotext.PDF(f)
    content = "\n\n".join(pdf)

    return {'abspath':os.path.abspath(file),'dirname':os.path.dirname(file),'basename':os.path.basename(file),'path0':os.path.dirname(file)+'/'+os.path.basename(file),'mtime':os.path.getmtime(file),'content':content}



#新增和更新数据库
def db_insert(file):
    #file='xxchinait/ip.txt'
    User = Query()
    #print(os.path.getmtime(file))
    db = TinyDB('knowledge.json')
    if db.search(User.mtime == os.path.getmtime(file)):
        print('pass',file)
        pass
    elif db.search(User.abspath==os.path.abspath(file)):#db.search(User.abspath=os.path.abspath(file)):
        db.update({'content':file2txt(file)['content'],'mtime':file2txt(file)['mtime']},User.abspath==os.path.abspath(file))
        #db.close()
        print('update',file)
    else:
        print('insert',file)
        db.insert(file2txt(file))
    #print(db.all())
    db.close()

#历遍目录
location='static'
def tree0(locationx):
    z=[]
    for root, dirs, files in os.walk(locationx, topdown=False):
        for name in files:
            path0=os.path.join(root, name).replace('\\','/')    
            if name.startswith('~'):
                pass
            elif name.endswith('.txt') or name.endswith('.TXT') or name.endswith('.docx') or name.endswith('.DOCX') or name.endswith('.xlsx') or name.endswith('.XLSX') or name.endswith('.pptx') or name.endswith('.PPTX') or name.endswith('.pdf'):
                if ' ' in name or ' ' in name:
                    print('*************',name,'**************')
                #print(path0)
                #txt2txt(path0)
                db_insert(path0)
                
                z.append(path0)
        for name in dirs:
            if ' ' in name or ' ' in name:
                print('*************',name,'**************')
            z.append(os.path.join(root, name).replace('\\','/'))
    z.sort()
    #print(z)
    return z



#删除旧的没有对应路径的数据库记录
def updatedb():
    list0=tree0(location)
    User = Query()
    db = TinyDB('knowledge.json')
    list1=list(map(lambda x:x['path0'],db.all()))

    for item in list1:
        if item in list0:
            pass
            #print(item,'yes')
        else:
            print(item,'not existed, will be remove')
            db.remove(User.abspath==os.path.abspath(item))
    db.close()

for root, dirs, files in os.walk(".", topdown=False):
    for name in files:
        for item in [' ',' ','(',')',',','（','）','，','。']:
            if item in name:
                try:
                    os.rename(os.path.join(root, name),os.path.join(root, name).replace(item,'_'))
                except:
                    pass
        #print(os.path.join(root, name))
    for name in dirs:
        for item in [' ',' ','(',')',',','（','）','，','。']:
            if item in name:
                try:
                    os.rename(os.path.join(root, name),os.path.join(root, name).replace(item,'_'))
                except:
                    pass
        #print(os.path.join(root, name))
updatedb()

